import datetime
import json
import traceback
from itertools import groupby
from operator import itemgetter

from fastapi import APIRouter, Request
from re_common.baselibrary.tools.all_requests.mrequest import MRequest
from re_common.baselibrary.utils.core.mlamada import bools_string
from re_common.baselibrary.utils.core.requests_core import INSIDE_HEADERS, MsgCode, SUCCESS

from apps.core.m_route import ContextIncludedRoute
from apps.core.return_info import ReturnInfo
from apps.crawler_platform.core_platform.core_nosql import NoSqlUtil
from apps.crawler_platform.core_platform.g_model import InputPlatformModel, \
    JournalArticleETLMode, JournalArticleETLValidateMode
from apps.crawler_platform.paper_platform.base_qk_step import BaseQKItems
from apps.crawler_platform.util.Mongodbhelper import MongoDBHelper
from apps.crawler_platform.util.etl_validate import EtlValidate
from apps.crawler_platform.util.nosqlhelper import NOSQLHelper
from apps.crawler_platform.util.requestapihelper import RequestApiHelper
from apps.sql_app.mmongodb import Coll
from settings import URLDISTRIBUTED

router = APIRouter(route_class=ContextIncludedRoute)


class EtlItems(BaseQKItems):
    def __init__(self):
        super(EtlItems, self).__init__()
        self.table: str = ''
        self.rawid: str = ''
        self.lngid: str = ''
        self.and_param: dict = {}
        #
        self.data: dict = {}
        self.mongo_html: dict = {}
        self.limit = 0

@router.post("/etl_api/validate_local")
async def etl_qk_validate_local(request: Request, input_data: InputPlatformModel[JournalArticleETLValidateMode]):
    """
        批量验证数据，通用验证数据
        :param request:
        :param input_data:
        :return: 返回的是验证失败的id和原因，由接口外程序剔除他们，如果没有错误，返回空的列表
        """
    return_info = ReturnInfo()
    list_data = input_data.data.list_data
    pm = EtlItems()
    redis = request.app.state.redis
    rs_data = {}
    for data in list_data:
        # 调用验证数据
        status, rsdic = await EtlValidate.validate_data_entry(redis, data)
        if not status:
            msg = rsdic["msg"]
            error_data = {
                "lngid": data["lngid"],
                "error_info": rsdic,
                "data": data
            }
            pm.data = error_data
            # 保存错误信息到 validate_error_latest
            bools, err_info = await NoSqlUtil.save_nosql_html_etl_validate_error(pm)
            if not bools:
                msg += ",错误字段信息保存至mongo失败"
            # 这里使用 data["rawid_mysql"]作为键已经考虑过数据分化的问题
            rs_data[data["keyid"]] = msg
    return_info.status = bools_string(True)
    return_info.msg_code = MsgCode.SUCCESS_CODE
    return_info.msg = "/etl_api/validate检测完成"
    return_info.data = rs_data
    return return_info.todict()


@router.post("/etl_api/validate")
async def etl_qk_validate(request: Request, input_data: InputPlatformModel[JournalArticleETLValidateMode]):
    """
    批量验证数据，通用验证数据
    :param request:
    :param input_data:
    :return: 返回的是验证失败的id和原因，由接口外程序剔除他们，如果没有错误，返回空的列表
    """
    return_info = ReturnInfo()
    list_data = input_data.data.list_data
    pm = EtlItems()
    redis = request.app.state.redis
    rs_data = {}
    for data in list_data:
        # 调用验证数据
        status, rsdic = await EtlValidate.validate_data_entry(redis, data)
        if not status:
            msg = rsdic["msg"]
            error_data = {
                "lngid": data["lngid"],
                "error_info": rsdic,
                "data": data
            }
            pm.data = error_data
            # 保存错误信息到 validate_error_latest
            bools, err_info = await NoSqlUtil.save_nosql_html_etl_validate_error(pm)
            if not bools:
                msg += ",错误字段信息保存至mongo失败"
            # 这里使用 data["rawid_mysql"]作为键已经考虑过数据分化的问题
            if "filename" in data.keys():
                msg += ",全文filename:{}".format(data["filename"])
            rs_data[data["rawid_mysql"]] = msg
    return_info.status = bools_string(True)
    return_info.msg_code = MsgCode.SUCCESS_CODE
    return_info.msg = "/etl_api/validate检测完成"
    return_info.data = rs_data
    return return_info.todict()


@router.post("/etl_api/batch/meta")
async def etl_update_batch(input: InputPlatformModel[JournalArticleETLMode]):
    """
    批量发送
    :param input:
    :return:
    """
    extra_sub_db_id = {
        "00002": ["00002", "00169", "00451", "00452"],
        "00004": ["00004", "00288"]
    }
    return_info = ReturnInfo()
    pm = EtlItems()
    pm.task_name = input.data.task_name
    pm.task_tag = input.data.task_tag
    # merge or overwrite
    op = input.data.op
    meta_data_dic = {}
    meta_data_list = []
    ref_data = {}
    fulltext_data = {}
    rs_data = {}
    # 一个列表的rawid(来自mysql数据库)
    list_rawid = input.data.rawid

    sub_db_id = input.data.sub_db_id
    # 查询latest表的数据
    if sub_db_id in extra_sub_db_id.keys():
        and_param = {"sub_db_id": {"$in": extra_sub_db_id[sub_db_id]}, 'rawid_mysql': {"$in": list_rawid}}
    else:
        and_param = {"sub_db_id": sub_db_id, "rawid_mysql": {"$in": list_rawid}}
    pm.and_param = and_param
    table_latest = input.data.table_latest
    pm.table = table_latest["meta"]
    print(pm)
    bools, err_info = await NoSqlUtil.read_nosql_latest(pm)
    if bools:
        meta_data_list = pm.mongo_html["data"]
    if not meta_data_list:
        return_info.status = bools_string(False)
        return_info.msg_code = MsgCode.MONGO_ERROR
        return_info.msg = "/etl_api/batch/meta接口查询laetst无数据"
        return_info.data = {}
        return return_info.todict()
    list_lngid = []
    # 找出没有在mongodb latest查到的数据
    for item in meta_data_list:  # 查latest的数据
        list_lngid.append(item["lngid"])
        if item["rawid_mysql"] in list_rawid:  # 传入的数据
            list_rawid.remove(item["rawid_mysql"])
    # 未在mongo查出来的,返回错误信息
    if len(list_rawid) > 0:
        for item in list_rawid:
            rs_data[item] = "/etl_api/batch/meta接口查询laetst无数据"
    if len(table_latest.get("ref","")) > 0:
        ref_pm = EtlItems()
        ref_pm.task_name = input.data.task_name
        ref_pm.task_tag = input.data.task_tag
        ref_pm.table = table_latest["ref"]
        ref_pm.and_param = {"lngid": {"$in": list_lngid}}
        bools, err_info = await NoSqlUtil.read_nosql_latest(ref_pm)
        if bools:
            ref_data = ref_pm.mongo_html["data"]
            for ritem in ref_data:
                for rli in ritem["refer_info"]:
                    if "repeat_cnt" in rli.keys():
                        rli.pop("repeat_cnt")
                    if "strtype_raw" in rli.keys():
                        rli.pop("strtype_raw")
                    if "ref_index" in rli.keys():
                        rli.pop("ref_index")
                    if "strtype_cn" in rli.keys():
                        rli.pop("strtype_cn")
                    if "refer_text_raw" in rli.keys():
                        rli.pop("refer_text_raw")
            # 这里的lngid是题录的lngid，多条引文在数据库中为1条
            ref_data.sort(key=itemgetter("lngid"), reverse=True)
            tmp_ref = groupby(ref_data, itemgetter('lngid'))
            ref_data = dict([(key, list(group)) for key, group in tmp_ref])
    if len(table_latest.get("fulltext", "")) > 0:
        fulltext_pm = EtlItems()
        fulltext_pm.task_name = input.data.task_name
        fulltext_pm.task_tag = input.data.task_tag
        fulltext_pm.table = table_latest["fulltext"]
        fulltext_pm.and_param = {"lngid": {"$in": list_lngid}}
        bools, err_info = await NoSqlUtil.read_nosql_latest(fulltext_pm)
        if bools:
            fulltext_data = fulltext_pm.mongo_html["data"]
            # 这里的lngid是题录的lngid，多条引文在数据库中为1条
            fulltext_data.sort(key=itemgetter("lngid"), reverse=True)
            tmp_fulltext = groupby(fulltext_data, itemgetter('lngid'))
            fulltext_data = dict([(key, list(group)) for key, group in tmp_fulltext])
    # 由于分化问题 这里会聚合出多条
    meta_data_list.sort(key=itemgetter("rawid_mysql"), reverse=True)
    tmp_meta = groupby(meta_data_list, itemgetter('rawid_mysql'))
    meta_data_dic = dict([(key, list(group)) for key, group in tmp_meta])
    # 批量检测
    validate_input = InputPlatformModel(
        data=JournalArticleETLValidateMode.parse_obj({"list_data": meta_data_list})).json()
    url = await URLDISTRIBUTED.get_etl_local_validate_url()
    rrq = RequestApiHelper.get_rrq()
    kwargs = {
        "rrq": rrq,
        "header": INSIDE_HEADERS,
        "url": url,
        "timeout": 240,
        "data": validate_input,
        "moths": MRequest.POST,
        "middler_list": [rrq.status_code_middlerwares,
                         rrq.is_null_html_middlerwares]
    }
    bools, dicts = await RequestApiHelper.etl_remote_meta(**kwargs)
    if not bools:
        return_info.status = bools_string(False)
        return_info.msg_code = MsgCode.STATUS_ERROR
        return_info.msg = "/etl_api/batch/meta调用验证接口失败"
        return_info.data = {}
        return return_info.todict()
    etl_meta = []
    etl_ref = []
    etl_fulltext = []
    # 根据批量检测回来的结果来删除有问题的题目不发送
    resp_dicts = json.loads(rrq.html)
    rs_check = resp_dicts.get("data", "")
    if len(rs_check) > 0:
        for k, v in rs_check.items():
            meta_data_dic.pop(k)
        rs_data.update(rs_check)
    if len(meta_data_dic.keys()) == 0:
        return_info.status = bools_string(False)
        return_info.msg_code = MsgCode.STATUS_ERROR
        return_info.msg = "/etl_api/batch/meta验证数据全部未通过"
        return_info.data = {}
        return return_info.todict()
    # 根据检测通过的题录列表，组装对应的引文,并删除题录中不需要发送的字段
    for rawid_mysql, metas in meta_data_dic.items():
        for m in metas:
            EtlValidate.etl_del(m)
            etl_meta.append(m)
            cur_ref = ref_data.get(m["lngid"])
            if cur_ref:
                etl_ref.extend(cur_ref)
            cur_fulltext = fulltext_data.get(m["lngid"])
            if cur_fulltext:
                for item in cur_fulltext:#将题录的rawid_mysql放进来,用来做检测聚合,后面发送前删除
                    item["rawid_mysql"] = rawid_mysql
                etl_fulltext.extend(cur_fulltext)
    # 远程etl
    url = await URLDISTRIBUTED.get_etl_remote_meta_batch_url()
    url = url.format(op)
    rrq = RequestApiHelper.get_rrq()
    kwargs = {
        "rrq": rrq,
        "header": INSIDE_HEADERS,
        "url": url,
        "timeout": 60,
        "data": json.dumps(etl_meta, ensure_ascii=False).strip(),
        "moths": MRequest.POST,
        "middler_list": [rrq.status_code_middlerwares,
                         rrq.is_null_html_middlerwares]
    }
    bools, dicts = await RequestApiHelper.etl_remote_meta(**kwargs)
    if not bools:
        return_info.status = bools_string(False)
        return_info.msg_code = MsgCode.STATUS_ERROR
        return_info.msg = "/etl_api/batch/meta调用远程etl接口失败:{}".format(sub_db_id)
        return_info.data = {}
        return return_info.todict()
    rjson = json.loads(rrq.html)
    if rjson["code"] != 1:
        return_info.status = bools_string(False)
        return_info.msg_code = rjson["code"]
        return_info.msg = "远程etl接口失败:" + rjson["error"]
        return_info.data = {}
        return return_info.todict()
    if len(etl_ref) > 0:
        url = await URLDISTRIBUTED.get_etl_remote_ref_batch_url()
        kwargs = {
            "rrq": rrq,
            "header": INSIDE_HEADERS,
            "url": url,
            "timeout": 60,
            "data": json.dumps(etl_ref, ensure_ascii=False).strip(),
            "moths": MRequest.POST,
            "middler_list": [rrq.status_code_middlerwares,
                             rrq.is_null_html_middlerwares]
        }
        bools, dicts = await RequestApiHelper.etl_remote_meta(**kwargs)
        if not bools:
            return_info.status = bools_string(False)
            return_info.msg_code = MsgCode.STATUS_ERROR
            return_info.msg = "/etl_api/batch/meta调用远程etl引文接口失败"
            return_info.data = {}
            return return_info.todict()
        rjson = json.loads(rrq.html)
        if rjson["code"] != 1:
            return_info.status = bools_string(False)
            return_info.msg_code = rjson["code"]
            return_info.msg = "/etl_api/batch/meta远程引文etl失败:" + rjson["error"]
            return_info.data = {}
            return return_info.todict()
    if len(etl_fulltext) > 0:
        etl_fulltext.sort(key=itemgetter("rawid_mysql"), reverse=True)
        tmp_fulltext = groupby(etl_fulltext, itemgetter('rawid_mysql'))
        fulltext_data = dict([(key, list(group)) for key, group in tmp_fulltext])
        # 批量检测全文
        validate_input = InputPlatformModel(
            data=JournalArticleETLValidateMode.parse_obj({"list_data": etl_fulltext})).json()
        url = await URLDISTRIBUTED.get_etl_local_validate_url()
        rrq = RequestApiHelper.get_rrq()
        kwargs = {
            "rrq": rrq,
            "header": INSIDE_HEADERS,
            "url": url,
            "timeout": 240,
            "data": validate_input,
            "moths": MRequest.POST,
            "middler_list": [rrq.status_code_middlerwares,
                             rrq.is_null_html_middlerwares]
        }
        bools, dicts = await RequestApiHelper.etl_remote_meta(**kwargs)
        if not bools:
            return_info.status = bools_string(False)
            return_info.msg_code = MsgCode.STATUS_ERROR
            return_info.msg = "/etl_api/batch/meta全文检测调用验证接口失败"
            return_info.data = {}
            return return_info.todict()
        # 根据批量检测回来的结果来删除有问题的全文不发送
        resp_dicts = json.loads(rrq.html)
        rs_check = resp_dicts.get("data", "")
        if len(rs_check) > 0:
            for k, v in rs_check.items():
                fulltext_data.pop(k)
            rs_data.update(rs_check)
        if len(fulltext_data.keys()) > 0:
            etl_fulltext = []
            for rawid_mysql, metas in fulltext_data.items():
                for m in metas:
                    EtlValidate.etl_del(m)
                    etl_fulltext.append(m)
            #发送检测通过的全文
            url = await URLDISTRIBUTED.get_etl_remote_fulltext_batch_url()
            kwargs = {
                "rrq": rrq,
                "header": INSIDE_HEADERS,
                "url": url,
                "timeout": 60,
                "data": json.dumps(etl_fulltext, ensure_ascii=False).strip(),
                "moths": MRequest.POST,
                "middler_list": [rrq.status_code_middlerwares,
                                 rrq.is_null_html_middlerwares]
            }
            bools, dicts = await RequestApiHelper.etl_remote_meta(**kwargs)
            if not bools:
                return_info.status = bools_string(False)
                return_info.msg_code = MsgCode.STATUS_ERROR
                return_info.msg = "/etl_api/batch/meta调用远程etl全文接口失败"
                return_info.data = {}
                return return_info.todict()
            rjson = json.loads(rrq.html)
            if rjson["code"] != 1:
                return_info.status = bools_string(False)
                return_info.msg_code = rjson["code"]
                return_info.msg = "/etl_api/batch/meta全文远程全文etl失败:" + rjson["error"]
                return_info.data = {}
                return return_info.todict()
    if len(etl_meta) > 0:
        etl_log_list = []
        for item in etl_meta:
            etl_log_list.append({
                "lngid": item["lngid"],
                "sub_db_id": item["sub_db_id"],
                "create_time": str(datetime.datetime.now())
            })
        mongo_conn = Coll.get_table_conn("etl_suc_logs")  # 根据table名取得mongo链接
        result = await NOSQLHelper.insert_many(mongo_conn, etl_log_list)
        # 记录日志 失败 无法通过rawid_mysql打标
        # if not bools:
        #     return_info.status = bools_string(False)
        #     return_info.msg_code = MsgCode.MONGO_ERROR
        #     return_info.msg = "/ietl_api/update/sngle 保存etl成功日志到mongo失败"
        #     return_info.data = {}
        #     return return_info.todict()
    return_info.status = SUCCESS
    return_info.msg_code = MsgCode.SUCCESS_CODE
    return_info.msg = "/etl_api/batch/meta调用成功"
    return_info.data = rs_data
    return return_info.todict()

@router.post("/etl_api/batch/ref")
async def etl_batch_ref(input: InputPlatformModel[JournalArticleETLMode]):
    """
    批量发送
    :param input:
    :return:
    """
    return_info = ReturnInfo()
    pm = EtlItems()
    pm.task_name = input.data.task_name
    pm.task_tag = input.data.task_tag
    meta_data_list = []
    ref_data = {}
    rs_data = {}
    # 一个列表的rawid(来自mysql数据库)
    list_rawid = input.data.rawid
    sub_db_id = input.data.sub_db_id
    and_param = {"sub_db_id": sub_db_id, "rawid_mysql": {"$in": list_rawid}}
    pm.and_param = and_param
    table_latest = input.data.table_latest
    pm.table = table_latest["meta"]
    bools, err_info = await NoSqlUtil.read_nosql_latest(pm)
    if bools:
        meta_data_list = pm.mongo_html["data"]
    if not meta_data_list:
        return_info.status = bools_string(False)
        return_info.msg_code = MsgCode.MONGO_ERROR
        return_info.msg = "/etl_api/batch/meta接口查询laetst无数据"
        return_info.data = {}
        return return_info.todict()
    list_lngid = []
    # 找出没有在mongodb latest查到的数据
    for item in meta_data_list:  # 查latest的数据
        list_lngid.append(item["lngid"])
        if item["rawid_mysql"] in list_rawid:  # 传入的数据
            list_rawid.remove(item["rawid_mysql"])
    etl_ref = []
    if len(table_latest.get("ref","")) > 0:
        ref_pm = EtlItems()
        ref_pm.task_name = input.data.task_name
        ref_pm.task_tag = input.data.task_tag
        ref_pm.table = table_latest["ref"]
        ref_pm.and_param = {"lngid": {"$in": list_lngid}}
        bools, err_info = await NoSqlUtil.read_nosql_latest(ref_pm)
        if bools:
            ref_data = ref_pm.mongo_html["data"]
            for ritem in ref_data:
                for rli in ritem["refer_info"]:
                    if "repeat_cnt" in rli.keys():
                        rli.pop("repeat_cnt")
                    if "strtype_raw" in rli.keys():
                        rli.pop("strtype_raw")
                    if "ref_index" in rli.keys():
                        rli.pop("ref_index")
                    if "strtype_cn" in rli.keys():
                        rli.pop("strtype_cn")
                    if "refer_text_raw" in rli.keys():
                        rli.pop("refer_text_raw")
    rrq = RequestApiHelper.get_rrq()
    # print(json.dumps(ref_data, ensure_ascii=False).strip())
    if len(ref_data) > 0:
        url = await URLDISTRIBUTED.get_etl_remote_ref_batch_url()
        kwargs = {
            "rrq": rrq,
            "header": INSIDE_HEADERS,
            "url": url,
            "timeout": 60,
            "data": json.dumps(ref_data, ensure_ascii=False).strip(),
            "moths": MRequest.POST,
            "middler_list": [rrq.status_code_middlerwares,
                             rrq.is_null_html_middlerwares]
        }
        bools, dicts = await RequestApiHelper.etl_remote_meta(**kwargs)
        if not bools:
            return_info.status = bools_string(False)
            return_info.msg_code = MsgCode.STATUS_ERROR
            return_info.msg = "/etl_api/batch/ref调用远程etl引文接口失败"
            return_info.data = {}
            return return_info.todict()
        rjson = json.loads(rrq.html)
        if rjson["code"] != 1:
            return_info.status = bools_string(False)
            return_info.msg_code = rjson["code"]
            return_info.msg = "/etl_api/batch/ref远程引文etl失败:" + rjson["error"]
            return_info.data = {}
            return return_info.todict()
    return_info.status = SUCCESS
    return_info.msg_code = MsgCode.SUCCESS_CODE
    return_info.msg = "/etl_api/batch/ref调用成功"
    return_info.data = rs_data
    return return_info.todict()